{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import catboost as cb\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn import preprocessing\n",
    "\n",
    "pd.set_option('display.max_colwidth', 60) #表中value值显示长度设为60\n",
    "pd.set_option('display.max_columns', 200)\n",
    "pd.set_option('display.max_rows', 200)\n",
    "\n",
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = pd.read_csv('./train/base_info.csv')\n",
    "label = pd.read_csv('./train/entprise_info.csv')\n",
    "base = pd.merge(base, label, on=['id'], how='left') #基本信息 标签\n",
    "\n",
    "annual=pd.read_csv('./train/annual_report_info.csv')\n",
    "annual=annual.drop_duplicates([\"id\"],keep='last')\n",
    "drop_1=[\"ANCHEYEAR\", \"MEMNUM\", \"FARNUM\", \"ANNNEWMEMNUM\", \"ANNREDMEMNUM\"]\n",
    "for p in drop_1:\n",
    "    del annual[p]\n",
    "base = pd.merge(base, annual, on=['id'], how='left')\n",
    "\n",
    "change = pd.read_csv(\"train/change_info.csv\")\n",
    "new_change = pd.DataFrame(change['id'].value_counts())\n",
    "new_change.columns=[\"bg_num\"]\n",
    "new_change[\"id\"]=new_change.index\n",
    "base = pd.merge(base, new_change, on=['id'], how='left')\n",
    "\n",
    "other = pd.read_csv(\"train/other_info.csv\")\n",
    "other=other.drop_duplicates([\"id\"],keep='last')\n",
    "base = pd.merge(base, other, on=['id'], how='left')\n",
    "\n",
    "tax = pd.read_csv(\"tax.csv\")\n",
    "base = pd.merge(base, tax, on=['id'], how='left')\n",
    "\n",
    "df = pd.read_csv(\"train/news_info.csv\")\n",
    "del df[\"public_date\"]\n",
    "positive = pd.DataFrame(df[df[\"positive_negtive\"]==\"积极\"][\"id\"].value_counts())\n",
    "positive.columns=[\"positive\"]\n",
    "positive[\"id\"]=positive.index\n",
    "\n",
    "negative = pd.DataFrame(df[df[\"positive_negtive\"]==\"消极\"][\"id\"].value_counts())\n",
    "negative.columns=[\"negative\"]\n",
    "negative[\"id\"]=negative.index\n",
    "\n",
    "neutral = pd.DataFrame(df[df[\"positive_negtive\"]==\"中立\"][\"id\"].value_counts())\n",
    "neutral.columns=[\"neutral\"]\n",
    "neutral[\"id\"]=neutral.index\n",
    "\n",
    "base = pd.merge(base, positive, on=['id'], how='left')\n",
    "base = pd.merge(base, negative, on=['id'], how='left')\n",
    "base = pd.merge(base, neutral, on=['id'], how='left')\n",
    "\n",
    "fill=[\"FUNDAM\", \"COLGRANUM\", \"RETSOLNUM\", \"DISPERNUM\", \"UNENUM\", \"COLEMPLNUM\", \"RETEMPLNUM\", \"DISEMPLNUM\", \n",
    "      \"UNEEMPLNUM\", \"bg_num\", \"legal_judgment_num\", \"brand_num\", \"patent_num\", \"positive\", \"negative\", \"neutral\", '个人所得税', \n",
    "      '印花税', '城市维护建设税', '营业税', '城镇土地使用税', '企业所得税', '土地增值税', '房产税', '水利建设专项收入', \n",
    "     '教育费附加', '地方教育附加', '契税', '税务部门罚没收入', '耕地占用税', '残疾人就业保障金', '其他收入', '其他专项收入']\n",
    "for p in fill:\n",
    "    base[p]=base[p].fillna(value=0)\n",
    "base[\"BUSSTNAME\"]=base[\"BUSSTNAME\"].fillna(value=\"开业\")\n",
    "\n",
    "#缺失值太多\n",
    "drop = ['enttypeitem', 'opto', 'empnum', 'compform', 'parnum',\n",
    "       'exenum', 'opform', 'ptbusscope', 'venind', 'enttypeminu',\n",
    "       'midpreindcode', 'protype', 'reccap', 'forreccap',\n",
    "       'forregcap', 'congro']\n",
    "\n",
    "for f in drop:\n",
    "    del base[f]\n",
    "\n",
    "del base['dom'], base['opscope'] #单一值太多\n",
    "del base['oploc']\n",
    "\n",
    "#拆分年月特征\n",
    "base['year'] = base['opfrom'].apply(lambda x: int(x.split('-')[0]))\n",
    "base['month'] = base['opfrom'].apply(lambda x: int(x.split('-')[1]))\n",
    "del base['opfrom']\n",
    "\n",
    "data = base.copy()\n",
    "\n",
    "num_feat = []\n",
    "cate_feat = []\n",
    "\n",
    "\n",
    "drop = ['id', 'label'] #不需要的特征\n",
    "cat = [\"industryphy\", \"BUSSTNAME\"] #类别特征\n",
    "for j in list(data.columns): \n",
    "    if j in drop:\n",
    "        continue\n",
    "    if j in cat:\n",
    "        cate_feat.append(j)\n",
    "    else:\n",
    "        num_feat.append(j)\n",
    "        \n",
    "for i in cate_feat:\n",
    "    data[i] = data[i].astype('category')\n",
    "features = num_feat + cate_feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 24 candidates, totalling 120 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  40 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=5)]: Done 120 out of 120 | elapsed: 23.6min finished\n",
      "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.2436253\ttest: 0.2436253\tbest: 0.2436253 (0)\ttotal: 17.3ms\tremaining: 20.7s\n",
      "75:\tlearn: 0.1356761\ttest: 0.1355603\tbest: 0.1355603 (75)\ttotal: 735ms\tremaining: 10.9s\n",
      "150:\tlearn: 0.1263998\ttest: 0.1263976\tbest: 0.1263976 (150)\ttotal: 1.39s\tremaining: 9.65s\n",
      "225:\tlearn: 0.1230072\ttest: 0.1232584\tbest: 0.1232584 (225)\ttotal: 2.03s\tremaining: 8.75s\n",
      "300:\tlearn: 0.1206344\ttest: 0.1210793\tbest: 0.1210793 (300)\ttotal: 2.76s\tremaining: 8.26s\n",
      "375:\tlearn: 0.1185706\ttest: 0.1192224\tbest: 0.1192224 (375)\ttotal: 3.4s\tremaining: 7.46s\n",
      "450:\tlearn: 0.1168203\ttest: 0.1177689\tbest: 0.1177689 (450)\ttotal: 4s\tremaining: 6.65s\n",
      "525:\tlearn: 0.1153217\ttest: 0.1165834\tbest: 0.1165834 (525)\ttotal: 4.54s\tremaining: 5.82s\n",
      "600:\tlearn: 0.1137190\ttest: 0.1152996\tbest: 0.1152996 (600)\ttotal: 5.12s\tremaining: 5.1s\n",
      "675:\tlearn: 0.1124255\ttest: 0.1143625\tbest: 0.1143625 (675)\ttotal: 5.67s\tremaining: 4.39s\n",
      "750:\tlearn: 0.1111178\ttest: 0.1133251\tbest: 0.1133251 (750)\ttotal: 6.26s\tremaining: 3.74s\n",
      "825:\tlearn: 0.1100153\ttest: 0.1124700\tbest: 0.1124700 (825)\ttotal: 6.83s\tremaining: 3.09s\n",
      "900:\tlearn: 0.1086970\ttest: 0.1114957\tbest: 0.1114957 (900)\ttotal: 7.41s\tremaining: 2.46s\n",
      "975:\tlearn: 0.1075532\ttest: 0.1104990\tbest: 0.1104990 (975)\ttotal: 7.98s\tremaining: 1.83s\n",
      "1050:\tlearn: 0.1061490\ttest: 0.1094542\tbest: 0.1094542 (1050)\ttotal: 8.55s\tremaining: 1.21s\n",
      "1125:\tlearn: 0.1050202\ttest: 0.1086161\tbest: 0.1086161 (1125)\ttotal: 9.13s\tremaining: 600ms\n",
      "1199:\tlearn: 0.1035734\ttest: 0.1073637\tbest: 0.1073637 (1199)\ttotal: 9.76s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.1073636722\n",
      "bestIteration = 1199\n",
      "\n",
      "参数的最佳取值:{'bagging_temperature': 0.2, 'border_count': 50, 'depth': 6, 'eval_metric': 'RMSE', 'iterations': 1200, 'l2_leaf_reg': 4, 'learning_rate': 0.03, 'metric_period': 75, 'od_type': 'Iter', 'od_wait': 100}\n",
      "最佳模型得分:0.7272011776577986\n"
     ]
    }
   ],
   "source": [
    "cb_model = cb.CatBoostRegressor(\n",
    "           cat_features=cate_feat\n",
    "           )\n",
    "\n",
    "if 'sample_weight' not in data.keys():\n",
    "        data['sample_weight'] = 1\n",
    "cb_model.random_state = 2018\n",
    "predict_label = 'predict_label'\n",
    "#kfold = KFold(n_splits=5, shuffle=True, random_state=2018)\n",
    "data[predict_label] = 0\n",
    "test_index = (data['label'].isnull()) | (data['label'] == -1)    #找到要预测的数据集\n",
    "train_data = data[~test_index].reset_index(drop=True)     #分割出预测集训练集\n",
    "test_data = data[test_index]\n",
    "\n",
    "\n",
    "train_x = train_data[features]\n",
    "train_y = train_data['label']\n",
    "test_x = train_data[features]\n",
    "test_y = train_data['label']\n",
    "\n",
    "# parameters = {'nthread':[2,4,5,6,7,8], #when use hyperthread, xgboost may become slower\n",
    "#               'objective':['reg:linear','reg:squarederror','reg:squaredlogerror','reg:logistic','reg:pseudohubererror'],\n",
    "#               'learning_rate': [.01,.03, 0.05], #so called `eta` value\n",
    "#               'max_depth': [7, 8, 9],\n",
    "#               'min_child_weight': [1,2,3,4,5,6]  ,\n",
    "#               'silent': [1],\n",
    "#               'subsample': [0.3,0.5,0.7,0.8],\n",
    "#               'colsample_bytree': [.5,.6,.7,.8],#the subsample ratio of columns when constructing each tree\n",
    "#               'n_estimators': [400,500,600,700],         \n",
    "#               }\n",
    "\n",
    "# parameters = {\n",
    "#     'max_depth': [6],\n",
    "#     'num_leaves':[40],\n",
    "#     'learning_rate': [0.05],\n",
    "#     'feature_fraction':[0.7],\n",
    "#     'min_child_samples':[18],\n",
    "#     'min_child_weight':[0.001],\n",
    "#     'colsample_bytree': [.5],\n",
    "#     'n_estimators': [400],\n",
    "#     'bagging_fraction': [1],\n",
    "#     'bagging_freq':[2],\n",
    "#     'reg_alpha':[0.001],\n",
    "#     'reg_lambda': [8],#0.01\n",
    "#     'num_iterations':[200]\n",
    "# }\n",
    "\n",
    "parameters = {'depth':[6,12],\n",
    "          'iterations':[700,800,900,1000,1100,1200],\n",
    "          'learning_rate':[0.03,0.02], \n",
    "          'l2_leaf_reg':[4],\n",
    "          'border_count':[50],\n",
    "          'eval_metric':['RMSE'],\n",
    "          'bagging_temperature': [0.2],\n",
    "          'od_type':['Iter'],\n",
    "          'metric_period':[75],\n",
    "          'od_wait':[100]\n",
    "          #'task_type':['GPU'],\n",
    "          #'ctr_border_count':[50,5,10,20,100,200]\n",
    "           }\n",
    "\n",
    "cb_grid = GridSearchCV(cb_model,\n",
    "                       parameters,\n",
    "                       cv = 5,#交叉验证数\n",
    "                       n_jobs = 5,\n",
    "                       verbose=True,\n",
    "                        )\n",
    "cb_grid.fit(train_x, train_y, eval_set=[(test_x, test_y)],\n",
    "          early_stopping_rounds=400,\n",
    "          #callbacks=[lgb.reset_parameter(learning_rate=lambda iter: max(0.005, 0.5 * (0.99 ** iter)))],\n",
    "          #categorical_feature=cate_feat,\n",
    "          sample_weight=train_data['sample_weight'],\n",
    "          verbose=75)\n",
    "print('参数的最佳取值:{0}'.format(cb_grid.best_params_))\n",
    "print('最佳模型得分:{0}'.format(cb_grid.best_score_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: Overfitting detector is active, thus evaluation metric is calculated on every iteration. 'metric_period' is ignored for evaluation metric.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.2451030\ttest: 0.2451030\tbest: 0.2451030 (0)\ttotal: 72.5ms\tremaining: 1m 12s\n",
      "75:\tlearn: 0.1384127\ttest: 0.1386718\tbest: 0.1386718 (75)\ttotal: 3.96s\tremaining: 48.1s\n",
      "150:\tlearn: 0.1163336\ttest: 0.1170764\tbest: 0.1170764 (150)\ttotal: 8.22s\tremaining: 46.2s\n",
      "225:\tlearn: 0.1059943\ttest: 0.1077965\tbest: 0.1077965 (225)\ttotal: 12.5s\tremaining: 42.9s\n",
      "300:\tlearn: 0.0992816\ttest: 0.1021812\tbest: 0.1021812 (300)\ttotal: 17s\tremaining: 39.5s\n",
      "375:\tlearn: 0.0933889\ttest: 0.0971522\tbest: 0.0971522 (375)\ttotal: 21.3s\tremaining: 35.3s\n",
      "450:\tlearn: 0.0886122\ttest: 0.0932089\tbest: 0.0932089 (450)\ttotal: 25.4s\tremaining: 31s\n",
      "525:\tlearn: 0.0833113\ttest: 0.0891208\tbest: 0.0891208 (525)\ttotal: 29.7s\tremaining: 26.8s\n",
      "600:\tlearn: 0.0786013\ttest: 0.0856068\tbest: 0.0856068 (600)\ttotal: 34.1s\tremaining: 22.6s\n",
      "675:\tlearn: 0.0748740\ttest: 0.0828120\tbest: 0.0828120 (675)\ttotal: 38.7s\tremaining: 18.6s\n",
      "750:\tlearn: 0.0714044\ttest: 0.0803804\tbest: 0.0803804 (750)\ttotal: 43s\tremaining: 14.3s\n",
      "825:\tlearn: 0.0696008\ttest: 0.0790136\tbest: 0.0790136 (825)\ttotal: 47s\tremaining: 9.91s\n",
      "900:\tlearn: 0.0667542\ttest: 0.0766217\tbest: 0.0766217 (900)\ttotal: 51.3s\tremaining: 5.64s\n",
      "975:\tlearn: 0.0643884\ttest: 0.0748191\tbest: 0.0748191 (975)\ttotal: 55.6s\tremaining: 1.37s\n",
      "999:\tlearn: 0.0637285\ttest: 0.0742620\tbest: 0.0742620 (999)\ttotal: 57s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.07426202401\n",
      "bestIteration = 999\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if 'sample_weight' not in data.keys():\n",
    "        data['sample_weight'] = 1\n",
    "predict_label = 'predict_label'\n",
    "#kfold = KFold(n_splits=5, shuffle=True, random_state=2018)\n",
    "data[predict_label] = 0\n",
    "test_index = (data['label'].isnull()) | (data['label'] == -1)    #找到要预测的数据集\n",
    "train_data = data[~test_index].reset_index(drop=True)     #分割出预测集训练集\n",
    "test_data = data[test_index]\n",
    "\n",
    "\n",
    "train_x = train_data[features]\n",
    "train_y = train_data['label']\n",
    "test_x = train_data[features]\n",
    "test_y = train_data['label']\n",
    "\n",
    "model = cb.CatBoostRegressor(iterations=1000,   #750\n",
    "                             learning_rate=0.02,   #0.02\n",
    "                             depth=12,   #12\n",
    "                             eval_metric='RMSE',\n",
    "                             random_seed = 23,\n",
    "                             bagging_temperature = 0.2,\n",
    "                             od_type='Iter',\n",
    "                             metric_period = 75,\n",
    "                             od_wait=100)\n",
    "model.fit(train_x, train_y, eval_set=[(test_x, test_y)], early_stopping_rounds=200,\n",
    "        # eval_metric='mae',\n",
    "        # callbacks=[lgb.reset_parameter(learning_rate=lambda iter: max(0.005, 0.5 * (0.99 ** iter)))],\n",
    "          cat_features=cate_feat,\n",
    "          sample_weight=train_data['sample_weight'],\n",
    "          verbose=75)\n",
    "\n",
    "data[predict_label] = model.predict(data[features])\n",
    "data['score'] = data[predict_label]\n",
    "df = data[data.label.isnull()][['id', 'score']]\n",
    "df['score'] = df['score'].apply(lambda x: 0 if x<0 else x) #修正\n",
    "df['score'] = df['score'].apply(lambda x: 1 if x>1 else x)\n",
    "\n",
    "#df=df.drop_duplicates([\"id\"])\n",
    "df.to_csv('result.csv', index=False) #submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
